package com.tangpian.sna.core.analysis.svm.pre;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;

import com.tangpian.sna.core.tokenizer.AnsjTokenizer;

import edu.udo.cs.wvtool.generic.tokenizer.WVTTokenizer;
import edu.udo.cs.wvtool.main.WVTDocumentInfo;
import edu.udo.cs.wvtool.util.TokenEnumeration;

public class VectorTokenizer extends AnsjTokenizer implements WVTTokenizer,
		TokenEnumeration {
	/** The underlying character stream of the currently tokenized document */
	private Reader input;

	/**
	 * The token, which is currently provided. This buffer is neccessary, to
	 * implement the semantic of TokenEnumeration
	 */
	private String currentToken;

	public VectorTokenizer() {
		input = null;
		currentToken = null;
	}

	/**
	 * @see edu.udo.cs.wvtool.generic.tokenizer.WVTTokenizer#tokenize(Reader,
	 *      WVTDocumentInfo)
	 */
	public TokenEnumeration tokenize(Reader source, WVTDocumentInfo d) {

		if (source != null) {
			BufferedReader bufferedReader = new BufferedReader(source);
			String line = null;
			StringBuffer stringBuffer = new StringBuffer();
			try {
				while ((line = bufferedReader.readLine()) != null) {
					if (!line.trim().isEmpty()) {
						stringBuffer.append(segmentation(line));
					}
				}
			} catch (IOException e1) {
				// TODO Auto-generated catch block
				e1.printStackTrace();
			}

			input = new StringReader(stringBuffer.toString());
			readNextToken();
			return this;
		} else {
			return null;
		}

	}

	/**
	 * Read a token from the character stream and store it into currentToken. If
	 * there are no more tokens left store a null value.
	 * 
	 */
	public void readNextToken() {

		StringBuffer buf = new StringBuffer();
		boolean endReached = false;
		int in = 0;

		try {

			// Read from the stream, until a letter occurs

			in = input.read();
			char ch = (char) in;

			while ((in != -1) && !Character.isLetter(ch)) {
				in = input.read();
				ch = (char) in;
			}

			if (in != -1)
				buf.append(ch);

			// Read from the stream, util a non-letter occurs

			while ((in != -1) && Character.isLetter(ch)) {

				in = input.read();
				ch = (char) in;

				if (Character.isLetter(ch))
					buf.append(ch);

			}
		} catch (Exception e) {
			endReached = true;

		}

		if (in == -1)
			endReached = true;

		if (endReached) {

			// If the stream ended with a non-empty token, this is the last
			// token, otherwise there is no more token.

			if (buf.length() > 0)
				currentToken = buf.toString();
			else
				currentToken = null;

			return;
		} else {

			// if the end of the stream has not been reached yet, simply store
			// the extracted token.
			currentToken = buf.toString();
			return;
		}
	}

	/**
	 * @see edu.udo.cs.wvtool.util.TokenEnumeration#hasMoreTokens()
	 */
	public boolean hasMoreTokens() {

		// If the current token does not equal the null value, then there is at
		// least this token left
		if (input != null)
			return (currentToken != null);
		else
			return false;
	}

	/**
	 * @see edu.udo.cs.wvtool.util.TokenEnumeration#nextToken()
	 */
	public String nextToken() {

		String result = null;

		// If unequal null, return the current token and read another one from
		// the stream

		if (currentToken != null) {
			result = currentToken;
			readNextToken();
		} else
			result = null;

		return result;
	}

	public String segmentation(String inputstring) {
		StringBuffer buffer = new StringBuffer();
		for (String string : tokenString(inputstring)) {
			buffer.append(string + " ");
		}
		return buffer.toString();
	}
}